# In order to run the code, uncomment this chunk and install required packages
# install.packages('readxl')
# install.packages('caret')
# install.packages('corrplot')
# install.packages('outliers')
# install.packages('e1071')
# install.packages("randomForest")
# install.packages("rpart.plot")
# install.packages("xgboost")
# install.packages("SHAPforxgboost")
# install.packages("factoextra")
# install.packages("NbClust")
# install.packages("plotly")
library("readxl")
library(corrplot)
library(outliers)
library(caret)
library(e1071)
library("randomForest")
library(rpart)
library(rpart.plot)
library(xgboost)
library("SHAPforxgboost")
library(factoextra)
library(NbClust)
library(clValid)
library(dplyr)
library(plotly)
# Loading data
data <- read_excel("ChurnData.xlsx")
New names:
• `` -> `...18`
head(data)
# Removing NA values
data <- data[,colSums(is.na(data))<nrow(data)]
data <- na.omit(data)
# Removing unnecessary column Customer
data <- subset(data, select = -Customer)
# Removing spaces in column names
colnames(data) <- make.names(names(data))
# Removing 0s in Account.Length column
data[data$Account.Length == 0, ] <- 1
hist(data$Customer.Service.Calls,
main = "Customer service calls distribution",
xlab = "Customer service calls, calls",
xlim = c(0, max(data$Customer.Service.Calls)),
col = "dodgerblue3",
breaks = 10)
axis(side=1, at=seq(0, 9, 1))

# Churners VS Non churners distribution
num_of_churners_non_churners <- c(nrow(data[data$Churn==1, ]),nrow(data[data$Churn==0, ]))
percentage <- round(num_of_churners_non_churners * 100 / sum(num_of_churners_non_churners), 1)
colors <- c("dodgerblue1","dodgerblue4")
pie(num_of_churners_non_churners,
main = "Churners VS Non churners distribution",
labels = percentage,
col = colors)
legend("topright", c("Churners","Non churners"), cex = 1, fill = colors)

par(mfrow=c(3,1))
# Income boxplot
boxplot(data$Income,
scientific = FALSE,
horizontal = TRUE,
ylim = c(0, max(data$Income)),
main = "Income distribution",
xlab = "Annual Income, $",
col="dodgerblue3")
# Account Length boxplot
boxplot(data$Account.Length,
scientific = FALSE,
horizontal = TRUE,
ylim = c(0, max(data$Account.Length)),
main = "Account Length distribution",
xlab = "Account Length, months",
col="dodgerblue3")
# Voice Mail Messages boxplot
boxplot(data$Voice.Mail.Messages,
scientific = FALSE,
horizontal = TRUE,
ylim = c(0, max(data$Voice.Mail.Messages)),
main = "Voice Mail Messages distribution",
xlab = "Voice Mail Messages, messages",
col="dodgerblue3")

par(mfrow=c(3,1))
# Day Minutes boxplot
boxplot(data$Day.Minutes,
scientific = FALSE,
horizontal = TRUE,
ylim = c(0, max(data$Day.Minutes)),
main = "Day Minutes distribution",
xlab = "Day Minutes, min",
col="dodgerblue3")
# Evening Minutes boxplot
boxplot(data$Evening.Minutes,
scientific = FALSE,
horizontal = TRUE,
ylim = c(0, max(data$Evening.Minutes)),
main = "Evening Minutes distribution",
xlab = "Evening Minutes, min",
col="dodgerblue3")
# Night Minutes boxplot
boxplot(data$Night.Minutes,
scientific = FALSE,
horizontal = TRUE,
ylim = c(0, max(data$Night.Minutes)),
main = "Night Minutes distribution",
xlab = "Night Minutes, min",
col="dodgerblue3")

# International Minutes boxplot
boxplot(data$International.Minutes,
scientific = FALSE,
horizontal = TRUE,
ylim = c(0, max(data$International.Minutes)),
main = "International Minutes distribution",
xlab = "International Minutes, min",
col="dodgerblue3")

data_out <- data[, c("Income", "Account.Length", "Day.Minutes", "Day.Calls", "Evening.Minutes", "Evening.Calls", "Night.Minutes", "Night.Calls", "Customer.Service.Calls")]
# Filtering out outliers using z-score method
z_scores <- abs(scale(data_out))
not_outliers <- which(!rowSums(z_scores>3))
data <- data[rownames(data) %in% not_outliers, ]
# Building the correlation plot to see if there is correlation between columns
corrplot(cor(data),
method = "circle",
type = "lower",
tl.col = "black",
tl.cex = 0.6,
bg = "white",
mar=c(0,0,2,0))

# Calculating monthly day, evening and night calls durations
data$dcall_avg <- round(data$Day.Minutes / data$Account.Length, 3)
data$ecall_avg <- round(data$Evening.Minutes / data$Account.Length, 3)
data$ncall_avg <- round(data$Night.Minutes / data$Account.Length, 3)
# Calculating average monthly day, evening and night calls durations for each plan type
dcalls_monthly_avg <- c(mean(data[data$Plan.Type==1, ]$dcall_avg), mean(data[data$Plan.Type==2, ]$dcall_avg), mean(data[data$Plan.Type==3, ]$dcall_avg), mean(data[data$Plan.Type==4, ]$dcall_avg), mean(data[data$Plan.Type==5, ]$dcall_avg), mean(data[data$Plan.Type==6, ]$dcall_avg), mean(data[data$Plan.Type==7, ]$dcall_avg), mean(data[data$Plan.Type==8, ]$dcall_avg), mean(data[data$Plan.Type==9, ]$dcall_avg), mean(data[data$Plan.Type==10, ]$dcall_avg))
ecalls_monthly_avg <- c(mean(data[data$Plan.Type==1, ]$ecall_avg), mean(data[data$Plan.Type==2, ]$ecall_avg), mean(data[data$Plan.Type==3, ]$ecall_avg), mean(data[data$Plan.Type==4, ]$ecall_avg), mean(data[data$Plan.Type==5, ]$ecall_avg), mean(data[data$Plan.Type==6, ]$ecall_avg), mean(data[data$Plan.Type==7, ]$ecall_avg), mean(data[data$Plan.Type==8, ]$ecall_avg), mean(data[data$Plan.Type==9, ]$ecall_avg), mean(data[data$Plan.Type==10, ]$ecall_avg))
ncalls_monthly_avg <- c(mean(data[data$Plan.Type==1, ]$ncall_avg), mean(data[data$Plan.Type==2, ]$ncall_avg), mean(data[data$Plan.Type==3, ]$ncall_avg), mean(data[data$Plan.Type==4, ]$ncall_avg), mean(data[data$Plan.Type==5, ]$ncall_avg), mean(data[data$Plan.Type==6, ]$ncall_avg), mean(data[data$Plan.Type==7, ]$ncall_avg), mean(data[data$Plan.Type==8, ]$ncall_avg), mean(data[data$Plan.Type==9, ]$ncall_avg), mean(data[data$Plan.Type==10, ]$ncall_avg))
# Visualizing average monthly day, evening and night calls durations for each plan type
plan_types <- c("1","2","3","4","5","6","7","8","9","10")
colors <- c("dodgerblue1","dodgerblue3","dodgerblue4")
day_time <- c("Day","Evening","Night")
calls_monthly_avg <- matrix(c(dcalls_monthly_avg, ecalls_monthly_avg, ncalls_monthly_avg), nrow = 3, ncol = 10, byrow = TRUE)
barplot(calls_monthly_avg,
main = "Average monthly calls duration by plan type",
names.arg = plan_types,
xlab = "Plan type",
ylab = "Average monthly calls duration, min",
col = colors)
legend("topright", day_time, cex = 0.6, fill = colors)

# Stratified random split
set.seed(123)
split_index <- createDataPartition(data$Churn, p = .7, list = FALSE, times = 1)
train <- data[split_index,]
test <- data[-split_index,]
train$Churn <- as.factor(train$Churn)
test$Churn <- as.factor(test$Churn)
# Creating additional variables to get rid of multicorrelation
data$dcall_dur_avg <- round(data$Day.Minutes / data$Day.Calls, 3)
data$ecall_dur_avg <- round(data$Evening.Minutes / data$Day.Calls, 3)
data$ncall_dur_avg <- round(data$Night.Minutes / data$Day.Calls, 3)
# Selecting features that will not be used for analysis
dropped_columns <- c("Voice.Mail","Day.Calls","Evening.Calls","Night.Calls","International.Plan.YES","International.Calls","Has.Phone","Phone.Payment.left","dcall_avg","ecall_avg","ncall_avg","dcall_dur_avg","ecall_dur_avg","ncall_dur_avg")
# Dropping unnecessary columns
train <- train[,!(names(train) %in% dropped_columns)]
test <- test[,!(names(test) %in% dropped_columns)]
# Creating dataframe that will later be used to build an ensamble algorithm
data_ens <- data[,!(names(data) %in% dropped_columns)]
# This formula will be used for all models
formula <- 'Churn ~ Income + Gender + Account.Length + Plan.Type + Voice.Mail.Messages + Day.Minutes + Evening.Minutes + Night.Minutes + International.Minutes + Phone.monthly.payment + Customer.Service.Calls'
# Logistic regression
logit <- glm(as.formula(formula),family=binomial(link='logit'),data=train)
summary(logit)
Call:
glm(formula = as.formula(formula), family = binomial(link = "logit"),
data = train)
Deviance Residuals:
Min 1Q Median 3Q Max
-1.7924 -0.5024 -0.3083 -0.1711 2.9670
Coefficients:
Estimate Std. Error z value Pr(>|z|)
(Intercept) -9.497e+00 8.894e-01 -10.678 < 2e-16 ***
Income 5.371e-06 3.475e-06 1.546 0.12218
Gender -9.708e-02 1.845e-01 -0.526 0.59880
Account.Length -2.052e-03 6.452e-03 -0.318 0.75043
Plan.Type -7.364e-03 3.196e-02 -0.230 0.81775
Voice.Mail.Messages -2.268e-02 7.954e-03 -2.851 0.00436 **
Day.Minutes 1.952e-02 1.942e-03 10.053 < 2e-16 ***
Evening.Minutes 8.412e-03 1.987e-03 4.232 2.31e-05 ***
Night.Minutes 4.695e-03 1.891e-03 2.482 0.01305 *
International.Minutes 1.434e-01 1.853e-02 7.736 1.02e-14 ***
Phone.monthly.payment -8.578e-03 4.287e-03 -2.001 0.04540 *
Customer.Service.Calls 4.353e-01 7.082e-02 6.147 7.90e-10 ***
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
(Dispersion parameter for binomial family taken to be 1)
Null deviance: 1019.99 on 1300 degrees of freedom
Residual deviance: 787.78 on 1289 degrees of freedom
AIC: 811.78
Number of Fisher Scoring iterations: 6
# Trained logistic regression predictions
model_fit <- predict(logit, test, type = 'response')
logit_pred <- ifelse(model_fit > 0.5,1,0)
# Confusion matrix for logistic regression
logit_conf <- confusionMatrix(as.factor(logit_pred), test$Churn, mode = "everything", positive = "1")
logit_conf
Confusion Matrix and Statistics
Reference
Prediction 0 1
0 475 56
1 10 16
Accuracy : 0.8815
95% CI : (0.8517, 0.9072)
No Information Rate : 0.8707
P-Value [Acc > NIR] : 0.2462
Kappa : 0.2769
Mcnemar's Test P-Value : 3.04e-08
Sensitivity : 0.22222
Specificity : 0.97938
Pos Pred Value : 0.61538
Neg Pred Value : 0.89454
Precision : 0.61538
Recall : 0.22222
F1 : 0.32653
Prevalence : 0.12926
Detection Rate : 0.02873
Detection Prevalence : 0.04668
Balanced Accuracy : 0.60080
'Positive' Class : 1
logit_f <- logit_conf$byClass[7]
# Algorithm to tune parameters of support vector machines
#
# tune_out <- tune.svm(x = train[, -12], y = train$Churn,
# type = "C-classification",
# kernel = "polynomial", degree = 2, cost = 10^(-1:1),
# gamma = c(0.1, 0.5, 1), coef0 = c(0.1, 0.5, 1))
#
#
# Obtained optimal values will further be used in svm
# tune_out$best.parameters$cost
# tune_out$best.parameters$gamma
# tune_out$best.parameters$coef0
set.seed(123)
# Support vector machines
svm_m = svm(formula = as.formula(formula), data = train, scale = TRUE, type = 'C-classification', kernel = "polynomial", degree = 2, cost = 0.1, gamma = 1, coef0 = 0.5)
set.seed(123)
# Support vector machines predictions
svm_p <- predict(svm_m, newdata = test)
svm_conf <- confusionMatrix(svm_p, test$Churn, mode = "everything", positive = "1")
svm_conf
Confusion Matrix and Statistics
Reference
Prediction 0 1
0 478 37
1 7 35
Accuracy : 0.921
95% CI : (0.8954, 0.942)
No Information Rate : 0.8707
P-Value [Acc > NIR] : 0.0001144
Kappa : 0.5734
Mcnemar's Test P-Value : 1.232e-05
Sensitivity : 0.48611
Specificity : 0.98557
Pos Pred Value : 0.83333
Neg Pred Value : 0.92816
Precision : 0.83333
Recall : 0.48611
F1 : 0.61404
Prevalence : 0.12926
Detection Rate : 0.06284
Detection Prevalence : 0.07540
Balanced Accuracy : 0.73584
'Positive' Class : 1
svm_f <- svm_conf$byClass[7]
# Tuning parameter mtry for random forest
set.seed(123)
train_rf <- as.data.frame(train)
bestmtry <- tuneRF(train_rf[, -12], train_rf[,12], stepFactor=1.5, improve=1e-5, ntree=500)
mtry = 3 OOB error = 8.46%
Searching left ...
mtry = 2 OOB error = 8.99%
-0.06363636 1e-05
Searching right ...
mtry = 4 OOB error = 8.69%
-0.02727273 1e-05

print(bestmtry)
mtry OOBError
2.OOB 2 0.08993082
3.OOB 3 0.08455035
4.OOB 4 0.08685626
set.seed(123)
# Random forest
randomf = randomForest(as.formula(formula), ntree = 500, mtry = 3, data = train)
set.seed(123)
# Random forest predictions
y_pred = predict(randomf, newdata = test[-12])
randf_conf <- confusionMatrix(y_pred, test$Churn, mode = "everything", positive = "1")
importance(randomf)
MeanDecreaseGini
Income 21.051915
Gender 3.779393
Account.Length 17.793486
Plan.Type 12.362275
Voice.Mail.Messages 10.860191
Day.Minutes 96.341691
Evening.Minutes 36.883533
Night.Minutes 26.307368
International.Minutes 26.356792
Phone.monthly.payment 15.413500
Customer.Service.Calls 32.591108
randf_conf
Confusion Matrix and Statistics
Reference
Prediction 0 1
0 479 32
1 6 40
Accuracy : 0.9318
95% CI : (0.9076, 0.9513)
No Information Rate : 0.8707
P-Value [Acc > NIR] : 2.423e-06
Kappa : 0.6419
Mcnemar's Test P-Value : 5.002e-05
Sensitivity : 0.55556
Specificity : 0.98763
Pos Pred Value : 0.86957
Neg Pred Value : 0.93738
Precision : 0.86957
Recall : 0.55556
F1 : 0.67797
Prevalence : 0.12926
Detection Rate : 0.07181
Detection Prevalence : 0.08259
Balanced Accuracy : 0.77159
'Positive' Class : 1
randf_f <- randf_conf$byClass[7]
set.seed(123)
# Simple decision treee
dtree <- rpart(formula = as.formula(formula), data = train, cp=0.01)
# Choosing optimal complexity parameter (cp) using plot
printcp(dtree)
Classification tree:
rpart(formula = as.formula(formula), data = train, cp = 0.01)
Variables actually used in tree construction:
[1] Customer.Service.Calls Day.Minutes Evening.Minutes International.Minutes
[5] Phone.monthly.payment Voice.Mail.Messages
Root node error: 173/1301 = 0.13297
n= 1301
CP nsplit rel error xerror xstd
1 0.167630 0 1.00000 1.00000 0.070793
2 0.063584 1 0.83237 0.87283 0.066781
3 0.057803 3 0.70520 0.82659 0.065214
4 0.023121 4 0.64740 0.74566 0.062312
5 0.011561 10 0.50867 0.80925 0.064609
6 0.010000 11 0.49711 0.79191 0.063996
plotcp(dtree)

rpart.plot(dtree)

set.seed(123)
# Devision tree predictions
tree_pred <- predict(dtree, test, type = 'class')
dtree_conf <- confusionMatrix(tree_pred, test$Churn, mode = "everything", positive = "1")
dtree_conf
Confusion Matrix and Statistics
Reference
Prediction 0 1
0 473 30
1 12 42
Accuracy : 0.9246
95% CI : (0.8994, 0.9451)
No Information Rate : 0.8707
P-Value [Acc > NIR] : 3.499e-05
Kappa : 0.6251
Mcnemar's Test P-Value : 0.008712
Sensitivity : 0.58333
Specificity : 0.97526
Pos Pred Value : 0.77778
Neg Pred Value : 0.94036
Precision : 0.77778
Recall : 0.58333
F1 : 0.66667
Prevalence : 0.12926
Detection Rate : 0.07540
Detection Prevalence : 0.09695
Balanced Accuracy : 0.77930
'Positive' Class : 1
dtree_f <- dtree_conf$byClass[7]
# Preparing data for xgboost algorithm
xgb_train_churn <- as.matrix(train[12])
xgb_train <- data.matrix(train[-12])
xgb_test_churn <- as.matrix(test[12])
xgb_test <- data.matrix(test[-12])
xgb_train_m = xgb.DMatrix(data=xgb_train, label=xgb_train_churn)
xgb_test_m = xgb.DMatrix(data=xgb_test, label=xgb_test_churn)
params <- list(booster = "gbtree", objective = "binary:logistic", eta=0.3, gamma=0, max_depth=6, min_child_weight=1, subsample=1, colsample_bytree=1)
# Extracting optimal features for xgboost using cross validation
xgbcv <- xgb.cv( params = params, data = xgb_train_m, nrounds = 100, nfold = 5, showsd = T, stratified = T, print.every.n = 10, early.stop.round = 20, maximize = T)
Warning: 'print.every.n' is deprecated.
Use 'print_every_n' instead.
See help("Deprecated") and help("xgboost-deprecated").
Warning: 'early.stop.round' is deprecated.
Use 'early_stopping_rounds' instead.
See help("Deprecated") and help("xgboost-deprecated").
[1] train-logloss:0.490664+0.002389 test-logloss:0.514700+0.002804
Multiple eval metrics are present. Will use test_logloss for early stopping.
Will train until test_logloss hasn't improved in 20 rounds.
[11] train-logloss:0.107408+0.004752 test-logloss:0.237669+0.016557
[21] train-logloss:0.060796+0.003263 test-logloss:0.240536+0.027036
Stopping. Best iteration:
[1] train-logloss:0.490664+0.002389 test-logloss:0.514700+0.002804
# XGBoost algorithm
set.seed(123)
xgb <- xgboost(data = xgb_train_m, params=params, nrounds=41)
[1] train-logloss:0.492093
[2] train-logloss:0.374901
[3] train-logloss:0.302414
[4] train-logloss:0.251825
[5] train-logloss:0.214501
[6] train-logloss:0.184347
[7] train-logloss:0.161494
[8] train-logloss:0.145188
[9] train-logloss:0.129774
[10] train-logloss:0.119050
[11] train-logloss:0.109443
[12] train-logloss:0.101285
[13] train-logloss:0.094453
[14] train-logloss:0.086692
[15] train-logloss:0.084022
[16] train-logloss:0.081549
[17] train-logloss:0.077827
[18] train-logloss:0.072342
[19] train-logloss:0.069526
[20] train-logloss:0.066059
[21] train-logloss:0.062552
[22] train-logloss:0.061327
[23] train-logloss:0.059917
[24] train-logloss:0.057204
[25] train-logloss:0.054538
[26] train-logloss:0.053387
[27] train-logloss:0.051132
[28] train-logloss:0.049806
[29] train-logloss:0.046619
[30] train-logloss:0.045243
[31] train-logloss:0.042654
[32] train-logloss:0.041777
[33] train-logloss:0.040314
[34] train-logloss:0.039193
[35] train-logloss:0.038492
[36] train-logloss:0.037844
[37] train-logloss:0.037258
[38] train-logloss:0.035513
[39] train-logloss:0.034067
[40] train-logloss:0.033318
[41] train-logloss:0.032010
set.seed(123)
# xgboost predictions
xgb_p = predict(xgb, xgb_test_m)
xgb_p = as.factor(round(xgb_p))
xgb_conf <- confusionMatrix(xgb_p, test$Churn, mode = "everything", positive = "1")
xgb_conf
Confusion Matrix and Statistics
Reference
Prediction 0 1
0 474 25
1 11 47
Accuracy : 0.9354
95% CI : (0.9116, 0.9543)
No Information Rate : 0.8707
P-Value [Acc > NIR] : 5.43e-07
Kappa : 0.687
Mcnemar's Test P-Value : 0.03026
Sensitivity : 0.65278
Specificity : 0.97732
Pos Pred Value : 0.81034
Neg Pred Value : 0.94990
Precision : 0.81034
Recall : 0.65278
F1 : 0.72308
Prevalence : 0.12926
Detection Rate : 0.08438
Detection Prevalence : 0.10413
Balanced Accuracy : 0.81505
'Positive' Class : 1
xgb_f <- xgb_conf$byClass[7]
# Importance of features
xgb.importance(model=xgb)
# SHaP values for each feature
shap <- shap.prep(xgb_model = xgb, X_train = xgb_train)
shap.plot.summary(shap)

shap.plot.dependence(shap, "Day.Minutes", alpha = 0.7, jitter_width = 0.1)
`geom_smooth()` using formula 'y ~ x'

shap.plot.dependence(shap, "Day.Minutes", color_feature = 'Evening.Minutes', alpha = 0.7, jitter_width = 0.1)
`geom_smooth()` using formula 'y ~ x'

shap.plot.dependence(shap, "Customer.Service.Calls", color_feature = 'Day.Minutes', alpha = 0.7, jitter_width = 0.1)
`geom_smooth()` using formula 'y ~ x'
Warning in simpleLoess(y, x, w, span, degree = degree, parametric = parametric, :
pseudoinverse used at -0.025
Warning in simpleLoess(y, x, w, span, degree = degree, parametric = parametric, :
neighborhood radius 2.025
Warning in simpleLoess(y, x, w, span, degree = degree, parametric = parametric, :
reciprocal condition number 5.302e-15
Warning in simpleLoess(y, x, w, span, degree = degree, parametric = parametric, :
There are other near singularities as well. 1

shap.plot.dependence(shap, "International.Minutes", color_feature = 'auto', alpha = 0.7, jitter_width = 0.1)
`geom_smooth()` using formula 'y ~ x'
Warning in simpleLoess(y, x, w, span, degree = degree, parametric = parametric, :
at -0.0945
Warning in simpleLoess(y, x, w, span, degree = degree, parametric = parametric, :
radius 0.0089303
Warning in simpleLoess(y, x, w, span, degree = degree, parametric = parametric, :
all data on boundary of neighborhood. make span bigger
Warning in simpleLoess(y, x, w, span, degree = degree, parametric = parametric, :
pseudoinverse used at -0.0945
Warning in simpleLoess(y, x, w, span, degree = degree, parametric = parametric, :
neighborhood radius 0.0945
Warning in simpleLoess(y, x, w, span, degree = degree, parametric = parametric, :
reciprocal condition number 1
Warning in simpleLoess(y, x, w, span, degree = degree, parametric = parametric, :
zero-width neighborhood. make span bigger
Warning: Computation failed in `stat_smooth()`:
NA/NaN/Inf in foreign function call (arg 5)

shap.plot.dependence(shap, "Evening.Minutes", color_feature = 'auto', alpha = 0.7, jitter_width = 0.1)
`geom_smooth()` using formula 'y ~ x'

set.seed(123)
# Predicting churn with each model
log_pred <- predict(logit, newdata = data_ens, type = 'response')
log_pred <- as.factor(ifelse(log_pred > 0.5,1,0))
svm_pred <- predict(svm_m, newdata = data_ens)
randf_pred <- predict(randomf, newdata = data_ens)
dtree_pred <- predict(dtree, newdata = data_ens, type = 'class')
xgb_matrix <- xgb.DMatrix(data=data.matrix(data_ens[,-12]), label=as.matrix(data_ens$Churn))
xgb_pred <- predict(xgb, newdata = xgb_matrix)
xgb_pred <- as.factor(round(xgb_pred))
# Building the dataframe for ensemble method
predictions <- data.frame(log_pred, svm_pred, randf_pred, dtree_pred, xgb_pred)
predictions$ens <- NA
# Assigning scores for each model based on respective F-score
sum_f <- sum(logit_f, svm_f, dtree_f, randf_f, xgb_f)
log_score <- (logit_f / sum_f)
svm_score <- (svm_f / sum_f)
randf_score <- (randf_f / sum_f)
dtree_score <- (dtree_f / sum_f)
xgb_score <- (xgb_f / sum_f)
scores <- c(log_score, svm_score, randf_score, dtree_score,xgb_score)
# Ensemble prediction function
for(i in 1:length(log_pred)){
y = 0
n = 0
for (j in 1:5){
if (predictions[i, j] == 0){
n = n + scores[j]
}else{
y = y + scores[j]
}
}
if (y > n){
predictions$ens[i] = 1
}else{
predictions$ens[i] = 0
}
}
# Measuring the performance of an ensemble
ens_conf <- confusionMatrix(as.factor(predictions$ens), as.factor(data_ens$Churn), mode = 'everything', positive = '1')
ens_conf
Confusion Matrix and Statistics
Reference
Prediction 0 1
0 1610 81
1 3 164
Accuracy : 0.9548
95% CI : (0.9443, 0.9638)
No Information Rate : 0.8681
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.7717
Mcnemar's Test P-Value : < 2.2e-16
Sensitivity : 0.66939
Specificity : 0.99814
Pos Pred Value : 0.98204
Neg Pred Value : 0.95210
Precision : 0.98204
Recall : 0.66939
F1 : 0.79612
Prevalence : 0.13186
Detection Rate : 0.08827
Detection Prevalence : 0.08988
Balanced Accuracy : 0.83376
'Positive' Class : 1
ens_f <- ens_conf$byClass[7]
f_scores_all <- c(logit_f, svm_f, dtree_f, randf_f, xgb_f, ens_f)
f_scores_all
F1 F1 F1 F1 F1 F1
0.3265306 0.6140351 0.6666667 0.6779661 0.7230769 0.7961165
# Min max normalization function
minmax_norm <- function(x) {
(x - min(x)) / (max(x) - min(x))
}
# Dataframe consisting only of people who churned
churn_df <- data_ens[data_ens$Churn==1, ]
churn_df <- churn_df[, c("Day.Minutes", "International.Minutes", "Customer.Service.Calls")]
# churn_df <- as.data.frame(lapply(churn_df, minmax_norm))
# churn_df <- churn_df[,!(names(churn_df) %in% c("Plan.Type", "Gender", "Churn"))]
churn_df <- as.data.frame(scale(churn_df))
# pca_res = prcomp(churn_df, center = TRUE, scale = TRUE)
# summary(pca_res)
# Visualizing elbow method
wss <- sapply(1:10, function(k){kmeans(churn_df, k, nstart=25,iter.max = 50 )$tot.withinss})
plot(1:10, wss,
type="b", pch = 19, frame = F,
xlab="Number of clusters K",
ylab="Total within-clusters sum of squares")

# Defining optimal number of k for k-means algorithm
set.seed(123)
# Silhouette method
fviz_nbclust(churn_df, kmeans, nstart = 25, k.max = 15, iter.max = 50, method = "silhouette")+
labs(subtitle = "Silhouette method")

# K-means with k=4
k4 <- kmeans(churn_df, centers = 4, iter.max = 50, nstart = 25)
k4
K-means clustering with 4 clusters of sizes 36, 115, 45, 49
Cluster means:
Day.Minutes International.Minutes Customer.Service.Calls
1 -1.0930388 1.3095389 0.4343530
2 0.6160607 -0.6653610 -0.4337388
3 0.4143949 1.3927979 -0.7551460
4 -1.0233746 -0.6796489 1.3923433
Clustering vector:
[1] 2 2 1 3 4 2 4 4 2 2 2 4 2 2 3 2 2 3 2 2 1 3 2 1 4 1 2 4 1 2 2 4 2 2 2 2 4 4 2 2 1 3 1 2 3 2 2 2 2 2 1 2
[53] 3 2 3 1 4 2 2 2 1 2 1 2 4 4 4 3 2 2 2 2 2 2 3 3 4 4 3 4 1 4 3 2 4 2 4 4 1 2 2 2 1 4 3 4 4 1 1 3 3 4 4 3
[105] 2 3 4 2 2 4 2 3 2 2 3 3 4 4 2 3 2 3 4 2 3 4 2 2 2 3 2 3 4 4 2 2 3 2 2 2 2 1 2 2 4 1 2 1 2 2 1 2 2 2 2 2
[157] 3 2 1 2 2 1 1 2 3 4 2 1 2 2 2 2 1 4 4 4 2 2 3 4 2 2 3 2 3 3 2 1 4 1 4 2 3 2 2 4 1 4 2 1 2 2 1 4 1 2 1 2
[209] 2 2 3 4 3 3 3 4 2 4 2 3 2 2 2 2 3 1 3 2 4 1 3 2 3 4 1 2 3 2 2 2 2 1 2 2 3
Within cluster sum of squares by cluster:
[1] 51.23809 90.22534 43.70436 29.47427
(between_SS / total_SS = 70.7 %)
Available components:
[1] "cluster" "centers" "totss" "withinss" "tot.withinss" "betweenss" "size"
[8] "iter" "ifault"
# Evaluating built clusters with silhouette width
sil <- silhouette(k4$cluster, dist(churn_df))
fviz_silhouette(sil)

churn_df$cluster = factor(k4$cluster)
p <- plot_ly(churn_df, x=~Day.Minutes, y=~International.Minutes,
z=~Customer.Service.Calls, color=~cluster) %>% add_markers(size=1.5)
print(p)
NULL
LS0tCnRpdGxlOiAiRGV2ZWxvcGluZyBhbiBlbnNlbWJsZSBhcHByb2FjaCBmb3IgcHJlZGljdGluZyBjdXN0b21lciBjaHVybiIKb3V0cHV0OiBodG1sX25vdGVib29rCmF1dGhvcjogTmF6YXIgVG9kb3NoY2h1awotLS0KCmBgYHtyfQojIEluIG9yZGVyIHRvIHJ1biB0aGUgY29kZSwgdW5jb21tZW50IHRoaXMgY2h1bmsgYW5kIGluc3RhbGwgcmVxdWlyZWQgcGFja2FnZXMKCiMgaW5zdGFsbC5wYWNrYWdlcygncmVhZHhsJykKIyBpbnN0YWxsLnBhY2thZ2VzKCdjYXJldCcpCiMgaW5zdGFsbC5wYWNrYWdlcygnY29ycnBsb3QnKQojIGluc3RhbGwucGFja2FnZXMoJ291dGxpZXJzJykKIyBpbnN0YWxsLnBhY2thZ2VzKCdlMTA3MScpCiMgaW5zdGFsbC5wYWNrYWdlcygicmFuZG9tRm9yZXN0IikKIyBpbnN0YWxsLnBhY2thZ2VzKCJycGFydC5wbG90IikKIyBpbnN0YWxsLnBhY2thZ2VzKCJ4Z2Jvb3N0IikKIyBpbnN0YWxsLnBhY2thZ2VzKCJTSEFQZm9yeGdib29zdCIpCiMgaW5zdGFsbC5wYWNrYWdlcygiZmFjdG9leHRyYSIpCiMgaW5zdGFsbC5wYWNrYWdlcygiTmJDbHVzdCIpCiMgaW5zdGFsbC5wYWNrYWdlcygicGxvdGx5IikKYGBgCgpgYGB7cn0KbGlicmFyeSgicmVhZHhsIikKbGlicmFyeShjb3JycGxvdCkKbGlicmFyeShvdXRsaWVycykKbGlicmFyeShjYXJldCkKbGlicmFyeShlMTA3MSkKbGlicmFyeSgicmFuZG9tRm9yZXN0IikKbGlicmFyeShycGFydCkKbGlicmFyeShycGFydC5wbG90KQpsaWJyYXJ5KHhnYm9vc3QpCmxpYnJhcnkoIlNIQVBmb3J4Z2Jvb3N0IikKbGlicmFyeShmYWN0b2V4dHJhKQpsaWJyYXJ5KE5iQ2x1c3QpCmxpYnJhcnkoY2xWYWxpZCkKbGlicmFyeShkcGx5cikKbGlicmFyeShwbG90bHkpCmBgYAoKYGBge3J9CiMgTG9hZGluZyBkYXRhCmRhdGEgPC0gcmVhZF9leGNlbCgiQ2h1cm5EYXRhLnhsc3giKQpoZWFkKGRhdGEpCmBgYAoKYGBge3J9CiMgUmVtb3ZpbmcgTkEgdmFsdWVzCmRhdGEgPC0gZGF0YVssY29sU3Vtcyhpcy5uYShkYXRhKSk8bnJvdyhkYXRhKV0KZGF0YSA8LSBuYS5vbWl0KGRhdGEpCiMgUmVtb3ZpbmcgdW5uZWNlc3NhcnkgY29sdW1uIEN1c3RvbWVyCmRhdGEgPC0gc3Vic2V0KGRhdGEsIHNlbGVjdCA9IC1DdXN0b21lcikKIyBSZW1vdmluZyBzcGFjZXMgaW4gY29sdW1uIG5hbWVzCmNvbG5hbWVzKGRhdGEpIDwtIG1ha2UubmFtZXMobmFtZXMoZGF0YSkpCiMgUmVtb3ZpbmcgMHMgaW4gQWNjb3VudC5MZW5ndGggY29sdW1uCmRhdGFbZGF0YSRBY2NvdW50Lkxlbmd0aCA9PSAwLCBdIDwtIDEKYGBgCgpgYGB7cn0KaGlzdChkYXRhJEN1c3RvbWVyLlNlcnZpY2UuQ2FsbHMsCiAgICBtYWluID0gIkN1c3RvbWVyIHNlcnZpY2UgY2FsbHMgZGlzdHJpYnV0aW9uIiwKICAgIHhsYWIgPSAiQ3VzdG9tZXIgc2VydmljZSBjYWxscywgY2FsbHMiLAogICAgeGxpbSA9IGMoMCwgbWF4KGRhdGEkQ3VzdG9tZXIuU2VydmljZS5DYWxscykpLAogICAgY29sID0gImRvZGdlcmJsdWUzIiwKICAgIGJyZWFrcyA9IDEwKQoKYXhpcyhzaWRlPTEsIGF0PXNlcSgwLCA5LCAxKSkKYGBgCgpgYGB7cn0KIyBDaHVybmVycyBWUyBOb24gY2h1cm5lcnMgZGlzdHJpYnV0aW9uCm51bV9vZl9jaHVybmVyc19ub25fY2h1cm5lcnMgPC0gYyhucm93KGRhdGFbZGF0YSRDaHVybj09MSwgXSksbnJvdyhkYXRhW2RhdGEkQ2h1cm49PTAsIF0pKQpwZXJjZW50YWdlIDwtIHJvdW5kKG51bV9vZl9jaHVybmVyc19ub25fY2h1cm5lcnMgKiAxMDAgLyBzdW0obnVtX29mX2NodXJuZXJzX25vbl9jaHVybmVycyksIDEpCmNvbG9ycyA8LSBjKCJkb2RnZXJibHVlMSIsImRvZGdlcmJsdWU0IikKCnBpZShudW1fb2ZfY2h1cm5lcnNfbm9uX2NodXJuZXJzLCAKICAgIG1haW4gPSAiQ2h1cm5lcnMgVlMgTm9uIGNodXJuZXJzIGRpc3RyaWJ1dGlvbiIsCiAgICBsYWJlbHMgPSBwZXJjZW50YWdlLAogICAgY29sID0gY29sb3JzKQoKbGVnZW5kKCJ0b3ByaWdodCIsIGMoIkNodXJuZXJzIiwiTm9uIGNodXJuZXJzIiksIGNleCA9IDEsIGZpbGwgPSBjb2xvcnMpCgpgYGAKCmBgYHtyfQpwYXIobWZyb3c9YygzLDEpKQojIEluY29tZSBib3hwbG90CmJveHBsb3QoZGF0YSRJbmNvbWUsCiAgICAgICAgc2NpZW50aWZpYyA9IEZBTFNFLAogICAgICAgIGhvcml6b250YWwgPSBUUlVFLAogICAgICAgIHlsaW0gPSBjKDAsIG1heChkYXRhJEluY29tZSkpLAogICAgICAgIG1haW4gPSAiSW5jb21lIGRpc3RyaWJ1dGlvbiIsCiAgICAgICAgeGxhYiA9ICJBbm51YWwgSW5jb21lLCAkIiwKICAgICAgICBjb2w9ImRvZGdlcmJsdWUzIikKIyBBY2NvdW50IExlbmd0aCBib3hwbG90CmJveHBsb3QoZGF0YSRBY2NvdW50Lkxlbmd0aCwKICAgICAgICBzY2llbnRpZmljID0gRkFMU0UsCiAgICAgICAgaG9yaXpvbnRhbCA9IFRSVUUsCiAgICAgICAgeWxpbSA9IGMoMCwgbWF4KGRhdGEkQWNjb3VudC5MZW5ndGgpKSwKICAgICAgICBtYWluID0gIkFjY291bnQgTGVuZ3RoIGRpc3RyaWJ1dGlvbiIsCiAgICAgICAgeGxhYiA9ICJBY2NvdW50IExlbmd0aCwgbW9udGhzIiwKICAgICAgICBjb2w9ImRvZGdlcmJsdWUzIikKIyBWb2ljZSBNYWlsIE1lc3NhZ2VzIGJveHBsb3QKYm94cGxvdChkYXRhJFZvaWNlLk1haWwuTWVzc2FnZXMsCiAgICAgICAgc2NpZW50aWZpYyA9IEZBTFNFLAogICAgICAgIGhvcml6b250YWwgPSBUUlVFLAogICAgICAgIHlsaW0gPSBjKDAsIG1heChkYXRhJFZvaWNlLk1haWwuTWVzc2FnZXMpKSwKICAgICAgICBtYWluID0gIlZvaWNlIE1haWwgTWVzc2FnZXMgZGlzdHJpYnV0aW9uIiwKICAgICAgICB4bGFiID0gIlZvaWNlIE1haWwgTWVzc2FnZXMsIG1lc3NhZ2VzIiwKICAgICAgICBjb2w9ImRvZGdlcmJsdWUzIikKYGBgCgpgYGB7cn0KcGFyKG1mcm93PWMoMywxKSkKIyBEYXkgTWludXRlcyBib3hwbG90CmJveHBsb3QoZGF0YSREYXkuTWludXRlcywKICAgICAgICBzY2llbnRpZmljID0gRkFMU0UsCiAgICAgICAgaG9yaXpvbnRhbCA9IFRSVUUsCiAgICAgICAgeWxpbSA9IGMoMCwgbWF4KGRhdGEkRGF5Lk1pbnV0ZXMpKSwKICAgICAgICBtYWluID0gIkRheSBNaW51dGVzIGRpc3RyaWJ1dGlvbiIsCiAgICAgICAgeGxhYiA9ICJEYXkgTWludXRlcywgbWluIiwKICAgICAgICBjb2w9ImRvZGdlcmJsdWUzIikKIyBFdmVuaW5nIE1pbnV0ZXMgYm94cGxvdApib3hwbG90KGRhdGEkRXZlbmluZy5NaW51dGVzLAogICAgICAgIHNjaWVudGlmaWMgPSBGQUxTRSwKICAgICAgICBob3Jpem9udGFsID0gVFJVRSwKICAgICAgICB5bGltID0gYygwLCBtYXgoZGF0YSRFdmVuaW5nLk1pbnV0ZXMpKSwKICAgICAgICBtYWluID0gIkV2ZW5pbmcgTWludXRlcyBkaXN0cmlidXRpb24iLAogICAgICAgIHhsYWIgPSAiRXZlbmluZyBNaW51dGVzLCBtaW4iLAogICAgICAgIGNvbD0iZG9kZ2VyYmx1ZTMiKQojIE5pZ2h0IE1pbnV0ZXMgYm94cGxvdApib3hwbG90KGRhdGEkTmlnaHQuTWludXRlcywKICAgICAgICBzY2llbnRpZmljID0gRkFMU0UsCiAgICAgICAgaG9yaXpvbnRhbCA9IFRSVUUsCiAgICAgICAgeWxpbSA9IGMoMCwgbWF4KGRhdGEkTmlnaHQuTWludXRlcykpLAogICAgICAgIG1haW4gPSAiTmlnaHQgTWludXRlcyBkaXN0cmlidXRpb24iLAogICAgICAgIHhsYWIgPSAiTmlnaHQgTWludXRlcywgbWluIiwKICAgICAgICBjb2w9ImRvZGdlcmJsdWUzIikKYGBgCgpgYGB7cn0KIyBJbnRlcm5hdGlvbmFsIE1pbnV0ZXMgYm94cGxvdApib3hwbG90KGRhdGEkSW50ZXJuYXRpb25hbC5NaW51dGVzLAogICAgICAgIHNjaWVudGlmaWMgPSBGQUxTRSwKICAgICAgICBob3Jpem9udGFsID0gVFJVRSwKICAgICAgICB5bGltID0gYygwLCBtYXgoZGF0YSRJbnRlcm5hdGlvbmFsLk1pbnV0ZXMpKSwKICAgICAgICBtYWluID0gIkludGVybmF0aW9uYWwgTWludXRlcyBkaXN0cmlidXRpb24iLAogICAgICAgIHhsYWIgPSAiSW50ZXJuYXRpb25hbCBNaW51dGVzLCBtaW4iLAogICAgICAgIGNvbD0iZG9kZ2VyYmx1ZTMiKQpgYGAKCmBgYHtyfQpkYXRhX291dCA8LSBkYXRhWywgYygiSW5jb21lIiwgIkFjY291bnQuTGVuZ3RoIiwgIkRheS5NaW51dGVzIiwgIkRheS5DYWxscyIsICJFdmVuaW5nLk1pbnV0ZXMiLCAiRXZlbmluZy5DYWxscyIsICJOaWdodC5NaW51dGVzIiwgIk5pZ2h0LkNhbGxzIiwgIkN1c3RvbWVyLlNlcnZpY2UuQ2FsbHMiKV0KCiMgRmlsdGVyaW5nIG91dCBvdXRsaWVycyB1c2luZyB6LXNjb3JlIG1ldGhvZAp6X3Njb3JlcyA8LSBhYnMoc2NhbGUoZGF0YV9vdXQpKQoKbm90X291dGxpZXJzIDwtIHdoaWNoKCFyb3dTdW1zKHpfc2NvcmVzPjMpKQoKZGF0YSA8LSBkYXRhW3Jvd25hbWVzKGRhdGEpICVpbiUgbm90X291dGxpZXJzLCBdCmBgYAoKYGBge3J9CiMgQnVpbGRpbmcgdGhlIGNvcnJlbGF0aW9uIHBsb3QgdG8gc2VlIGlmIHRoZXJlIGlzIGNvcnJlbGF0aW9uIGJldHdlZW4gY29sdW1ucwpjb3JycGxvdChjb3IoZGF0YSksCiAgICAgICAgIG1ldGhvZCA9ICJjaXJjbGUiLAogICAgICAgICB0eXBlID0gImxvd2VyIiwKICAgICAgICAgdGwuY29sID0gImJsYWNrIiwKICAgICAgICAgdGwuY2V4ID0gMC42LAogICAgICAgICBiZyA9ICJ3aGl0ZSIsCiAgICAgICAgIG1hcj1jKDAsMCwyLDApKQpgYGAKCmBgYHtyfQojIENhbGN1bGF0aW5nIG1vbnRobHkgZGF5LCBldmVuaW5nIGFuZCBuaWdodCBjYWxscyBkdXJhdGlvbnMKZGF0YSRkY2FsbF9hdmcgPC0gcm91bmQoZGF0YSREYXkuTWludXRlcyAvIGRhdGEkQWNjb3VudC5MZW5ndGgsIDMpCmRhdGEkZWNhbGxfYXZnIDwtIHJvdW5kKGRhdGEkRXZlbmluZy5NaW51dGVzIC8gZGF0YSRBY2NvdW50Lkxlbmd0aCwgMykKZGF0YSRuY2FsbF9hdmcgPC0gcm91bmQoZGF0YSROaWdodC5NaW51dGVzIC8gZGF0YSRBY2NvdW50Lkxlbmd0aCwgMykKYGBgCgpgYGB7cn0KIyBDYWxjdWxhdGluZyBhdmVyYWdlIG1vbnRobHkgZGF5LCBldmVuaW5nIGFuZCBuaWdodCBjYWxscyBkdXJhdGlvbnMgZm9yIGVhY2ggcGxhbiB0eXBlCmRjYWxsc19tb250aGx5X2F2ZyA8LSBjKG1lYW4oZGF0YVtkYXRhJFBsYW4uVHlwZT09MSwgXSRkY2FsbF9hdmcpLCBtZWFuKGRhdGFbZGF0YSRQbGFuLlR5cGU9PTIsIF0kZGNhbGxfYXZnKSwgbWVhbihkYXRhW2RhdGEkUGxhbi5UeXBlPT0zLCBdJGRjYWxsX2F2ZyksIG1lYW4oZGF0YVtkYXRhJFBsYW4uVHlwZT09NCwgXSRkY2FsbF9hdmcpLCBtZWFuKGRhdGFbZGF0YSRQbGFuLlR5cGU9PTUsIF0kZGNhbGxfYXZnKSwgbWVhbihkYXRhW2RhdGEkUGxhbi5UeXBlPT02LCBdJGRjYWxsX2F2ZyksIG1lYW4oZGF0YVtkYXRhJFBsYW4uVHlwZT09NywgXSRkY2FsbF9hdmcpLCBtZWFuKGRhdGFbZGF0YSRQbGFuLlR5cGU9PTgsIF0kZGNhbGxfYXZnKSwgbWVhbihkYXRhW2RhdGEkUGxhbi5UeXBlPT05LCBdJGRjYWxsX2F2ZyksIG1lYW4oZGF0YVtkYXRhJFBsYW4uVHlwZT09MTAsIF0kZGNhbGxfYXZnKSkKCmVjYWxsc19tb250aGx5X2F2ZyA8LSBjKG1lYW4oZGF0YVtkYXRhJFBsYW4uVHlwZT09MSwgXSRlY2FsbF9hdmcpLCBtZWFuKGRhdGFbZGF0YSRQbGFuLlR5cGU9PTIsIF0kZWNhbGxfYXZnKSwgbWVhbihkYXRhW2RhdGEkUGxhbi5UeXBlPT0zLCBdJGVjYWxsX2F2ZyksIG1lYW4oZGF0YVtkYXRhJFBsYW4uVHlwZT09NCwgXSRlY2FsbF9hdmcpLCBtZWFuKGRhdGFbZGF0YSRQbGFuLlR5cGU9PTUsIF0kZWNhbGxfYXZnKSwgbWVhbihkYXRhW2RhdGEkUGxhbi5UeXBlPT02LCBdJGVjYWxsX2F2ZyksIG1lYW4oZGF0YVtkYXRhJFBsYW4uVHlwZT09NywgXSRlY2FsbF9hdmcpLCBtZWFuKGRhdGFbZGF0YSRQbGFuLlR5cGU9PTgsIF0kZWNhbGxfYXZnKSwgbWVhbihkYXRhW2RhdGEkUGxhbi5UeXBlPT05LCBdJGVjYWxsX2F2ZyksIG1lYW4oZGF0YVtkYXRhJFBsYW4uVHlwZT09MTAsIF0kZWNhbGxfYXZnKSkKCm5jYWxsc19tb250aGx5X2F2ZyA8LSBjKG1lYW4oZGF0YVtkYXRhJFBsYW4uVHlwZT09MSwgXSRuY2FsbF9hdmcpLCBtZWFuKGRhdGFbZGF0YSRQbGFuLlR5cGU9PTIsIF0kbmNhbGxfYXZnKSwgbWVhbihkYXRhW2RhdGEkUGxhbi5UeXBlPT0zLCBdJG5jYWxsX2F2ZyksIG1lYW4oZGF0YVtkYXRhJFBsYW4uVHlwZT09NCwgXSRuY2FsbF9hdmcpLCBtZWFuKGRhdGFbZGF0YSRQbGFuLlR5cGU9PTUsIF0kbmNhbGxfYXZnKSwgbWVhbihkYXRhW2RhdGEkUGxhbi5UeXBlPT02LCBdJG5jYWxsX2F2ZyksIG1lYW4oZGF0YVtkYXRhJFBsYW4uVHlwZT09NywgXSRuY2FsbF9hdmcpLCBtZWFuKGRhdGFbZGF0YSRQbGFuLlR5cGU9PTgsIF0kbmNhbGxfYXZnKSwgbWVhbihkYXRhW2RhdGEkUGxhbi5UeXBlPT05LCBdJG5jYWxsX2F2ZyksIG1lYW4oZGF0YVtkYXRhJFBsYW4uVHlwZT09MTAsIF0kbmNhbGxfYXZnKSkKYGBgCgpgYGB7cn0KIyBWaXN1YWxpemluZyBhdmVyYWdlIG1vbnRobHkgZGF5LCBldmVuaW5nIGFuZCBuaWdodCBjYWxscyBkdXJhdGlvbnMgZm9yIGVhY2ggcGxhbiB0eXBlCnBsYW5fdHlwZXMgPC0gYygiMSIsIjIiLCIzIiwiNCIsIjUiLCI2IiwiNyIsIjgiLCI5IiwiMTAiKQpjb2xvcnMgPC0gYygiZG9kZ2VyYmx1ZTEiLCJkb2RnZXJibHVlMyIsImRvZGdlcmJsdWU0IikKZGF5X3RpbWUgPC0gYygiRGF5IiwiRXZlbmluZyIsIk5pZ2h0IikKCmNhbGxzX21vbnRobHlfYXZnIDwtIG1hdHJpeChjKGRjYWxsc19tb250aGx5X2F2ZywgZWNhbGxzX21vbnRobHlfYXZnLCBuY2FsbHNfbW9udGhseV9hdmcpLCBucm93ID0gMywgbmNvbCA9IDEwLCBieXJvdyA9IFRSVUUpCgpiYXJwbG90KGNhbGxzX21vbnRobHlfYXZnLCAKICAgICAgICBtYWluID0gIkF2ZXJhZ2UgbW9udGhseSBjYWxscyBkdXJhdGlvbiBieSBwbGFuIHR5cGUiLCAKICAgICAgICBuYW1lcy5hcmcgPSBwbGFuX3R5cGVzLCAKICAgICAgICB4bGFiID0gIlBsYW4gdHlwZSIsIAogICAgICAgIHlsYWIgPSAiQXZlcmFnZSBtb250aGx5IGNhbGxzIGR1cmF0aW9uLCBtaW4iLCAKICAgICAgICBjb2wgPSBjb2xvcnMpCgpsZWdlbmQoInRvcHJpZ2h0IiwgZGF5X3RpbWUsIGNleCA9IDAuNiwgZmlsbCA9IGNvbG9ycykKCmBgYAoKYGBge3J9CiMgU3RyYXRpZmllZCByYW5kb20gc3BsaXQKc2V0LnNlZWQoMTIzKQpzcGxpdF9pbmRleCA8LSBjcmVhdGVEYXRhUGFydGl0aW9uKGRhdGEkQ2h1cm4sIHAgPSAuNywgbGlzdCA9IEZBTFNFLCB0aW1lcyA9IDEpCnRyYWluIDwtIGRhdGFbc3BsaXRfaW5kZXgsXQp0ZXN0IDwtIGRhdGFbLXNwbGl0X2luZGV4LF0KCnRyYWluJENodXJuIDwtIGFzLmZhY3Rvcih0cmFpbiRDaHVybikKdGVzdCRDaHVybiA8LSBhcy5mYWN0b3IodGVzdCRDaHVybikKYGBgCgpgYGB7cn0KIyBDcmVhdGluZyBhZGRpdGlvbmFsIHZhcmlhYmxlcyB0byBnZXQgcmlkIG9mIG11bHRpY29ycmVsYXRpb24KZGF0YSRkY2FsbF9kdXJfYXZnIDwtIHJvdW5kKGRhdGEkRGF5Lk1pbnV0ZXMgLyBkYXRhJERheS5DYWxscywgMykKZGF0YSRlY2FsbF9kdXJfYXZnIDwtIHJvdW5kKGRhdGEkRXZlbmluZy5NaW51dGVzIC8gZGF0YSREYXkuQ2FsbHMsIDMpCmRhdGEkbmNhbGxfZHVyX2F2ZyA8LSByb3VuZChkYXRhJE5pZ2h0Lk1pbnV0ZXMgLyBkYXRhJERheS5DYWxscywgMykKYGBgCgpgYGB7cn0KIyBTZWxlY3RpbmcgZmVhdHVyZXMgdGhhdCB3aWxsIG5vdCBiZSB1c2VkIGZvciBhbmFseXNpcwpkcm9wcGVkX2NvbHVtbnMgPC0gYygiVm9pY2UuTWFpbCIsIkRheS5DYWxscyIsIkV2ZW5pbmcuQ2FsbHMiLCJOaWdodC5DYWxscyIsIkludGVybmF0aW9uYWwuUGxhbi5ZRVMiLCJJbnRlcm5hdGlvbmFsLkNhbGxzIiwiSGFzLlBob25lIiwiUGhvbmUuUGF5bWVudC5sZWZ0IiwiZGNhbGxfYXZnIiwiZWNhbGxfYXZnIiwibmNhbGxfYXZnIiwiZGNhbGxfZHVyX2F2ZyIsImVjYWxsX2R1cl9hdmciLCJuY2FsbF9kdXJfYXZnIikKCiMgRHJvcHBpbmcgdW5uZWNlc3NhcnkgY29sdW1ucwp0cmFpbiA8LSB0cmFpblssIShuYW1lcyh0cmFpbikgJWluJSBkcm9wcGVkX2NvbHVtbnMpXQp0ZXN0IDwtIHRlc3RbLCEobmFtZXModGVzdCkgJWluJSBkcm9wcGVkX2NvbHVtbnMpXQoKIyBDcmVhdGluZyBkYXRhZnJhbWUgdGhhdCB3aWxsIGxhdGVyIGJlIHVzZWQgdG8gYnVpbGQgYW4gZW5zYW1ibGUgYWxnb3JpdGhtCmRhdGFfZW5zIDwtIGRhdGFbLCEobmFtZXMoZGF0YSkgJWluJSBkcm9wcGVkX2NvbHVtbnMpXQpgYGAKCmBgYHtyfQojIFRoaXMgZm9ybXVsYSB3aWxsIGJlIHVzZWQgZm9yIGFsbCBtb2RlbHMKZm9ybXVsYSA8LSAnQ2h1cm4gfiBJbmNvbWUgKyBHZW5kZXIgKyBBY2NvdW50Lkxlbmd0aCArIFBsYW4uVHlwZSArIFZvaWNlLk1haWwuTWVzc2FnZXMgKyBEYXkuTWludXRlcyArIEV2ZW5pbmcuTWludXRlcyArIE5pZ2h0Lk1pbnV0ZXMgKyBJbnRlcm5hdGlvbmFsLk1pbnV0ZXMgKyBQaG9uZS5tb250aGx5LnBheW1lbnQgKyBDdXN0b21lci5TZXJ2aWNlLkNhbGxzJwpgYGAKCmBgYHtyfQojIExvZ2lzdGljIHJlZ3Jlc3Npb24KbG9naXQgPC0gZ2xtKGFzLmZvcm11bGEoZm9ybXVsYSksZmFtaWx5PWJpbm9taWFsKGxpbms9J2xvZ2l0JyksZGF0YT10cmFpbikKc3VtbWFyeShsb2dpdCkKYGBgCgpgYGB7cn0KIyBUcmFpbmVkIGxvZ2lzdGljIHJlZ3Jlc3Npb24gcHJlZGljdGlvbnMKbW9kZWxfZml0IDwtIHByZWRpY3QobG9naXQsIHRlc3QsIHR5cGUgPSAncmVzcG9uc2UnKQpsb2dpdF9wcmVkIDwtIGlmZWxzZShtb2RlbF9maXQgPiAwLjUsMSwwKQojIENvbmZ1c2lvbiBtYXRyaXggZm9yIGxvZ2lzdGljIHJlZ3Jlc3Npb24KbG9naXRfY29uZiA8LSBjb25mdXNpb25NYXRyaXgoYXMuZmFjdG9yKGxvZ2l0X3ByZWQpLCB0ZXN0JENodXJuLCBtb2RlID0gImV2ZXJ5dGhpbmciLCBwb3NpdGl2ZSA9ICIxIikKbG9naXRfY29uZgpsb2dpdF9mIDwtIGxvZ2l0X2NvbmYkYnlDbGFzc1s3XQpgYGAKCmBgYHtyfQojIEFsZ29yaXRobSB0byB0dW5lIHBhcmFtZXRlcnMgb2Ygc3VwcG9ydCB2ZWN0b3IgbWFjaGluZXMKIyAKIyB0dW5lX291dCA8LSB0dW5lLnN2bSh4ID0gdHJhaW5bLCAtMTJdLCB5ID0gdHJhaW4kQ2h1cm4sCiMgICAgICAgICAgICAgICB0eXBlID0gIkMtY2xhc3NpZmljYXRpb24iLAojICAgICAgICAgICAgICAga2VybmVsID0gInBvbHlub21pYWwiLCBkZWdyZWUgPSAyLCBjb3N0ID0gMTBeKC0xOjEpLAojICAgICAgICAgICAgICAgZ2FtbWEgPSBjKDAuMSwgMC41LCAxKSwgY29lZjAgPSBjKDAuMSwgMC41LCAxKSkKIyAKIyAKIyBPYnRhaW5lZCBvcHRpbWFsIHZhbHVlcyB3aWxsIGZ1cnRoZXIgYmUgdXNlZCBpbiBzdm0KIyB0dW5lX291dCRiZXN0LnBhcmFtZXRlcnMkY29zdAojIHR1bmVfb3V0JGJlc3QucGFyYW1ldGVycyRnYW1tYQojIHR1bmVfb3V0JGJlc3QucGFyYW1ldGVycyRjb2VmMApgYGAKCmBgYHtyfQpzZXQuc2VlZCgxMjMpCiMgU3VwcG9ydCB2ZWN0b3IgbWFjaGluZXMKc3ZtX20gPSBzdm0oZm9ybXVsYSA9IGFzLmZvcm11bGEoZm9ybXVsYSksIGRhdGEgPSB0cmFpbiwgc2NhbGUgPSBUUlVFLCB0eXBlID0gJ0MtY2xhc3NpZmljYXRpb24nLCAga2VybmVsID0gInBvbHlub21pYWwiLCBkZWdyZWUgPSAyLCBjb3N0ID0gMC4xLCAgZ2FtbWEgPSAxLCBjb2VmMCA9IDAuNSkKYGBgCgpgYGB7cn0Kc2V0LnNlZWQoMTIzKQojIFN1cHBvcnQgdmVjdG9yIG1hY2hpbmVzIHByZWRpY3Rpb25zCnN2bV9wIDwtIHByZWRpY3Qoc3ZtX20sIG5ld2RhdGEgPSB0ZXN0KQpzdm1fY29uZiA8LSBjb25mdXNpb25NYXRyaXgoc3ZtX3AsIHRlc3QkQ2h1cm4sIG1vZGUgPSAiZXZlcnl0aGluZyIsIHBvc2l0aXZlID0gIjEiKQpzdm1fY29uZgpzdm1fZiA8LSBzdm1fY29uZiRieUNsYXNzWzddCmBgYApgYGB7cn0KIyBUdW5pbmcgcGFyYW1ldGVyIG10cnkgZm9yIHJhbmRvbSBmb3Jlc3QKc2V0LnNlZWQoMTIzKQp0cmFpbl9yZiA8LSBhcy5kYXRhLmZyYW1lKHRyYWluKQpiZXN0bXRyeSA8LSB0dW5lUkYodHJhaW5fcmZbLCAtMTJdLCB0cmFpbl9yZlssMTJdLCBzdGVwRmFjdG9yPTEuNSwgaW1wcm92ZT0xZS01LCBudHJlZT01MDApCnByaW50KGJlc3RtdHJ5KQpgYGAKCgpgYGB7cn0Kc2V0LnNlZWQoMTIzKQojIFJhbmRvbSBmb3Jlc3QKcmFuZG9tZiA9IHJhbmRvbUZvcmVzdChhcy5mb3JtdWxhKGZvcm11bGEpLCBudHJlZSA9IDUwMCwgIG10cnkgPSAzLCBkYXRhID0gdHJhaW4pCmBgYAoKYGBge3J9CnNldC5zZWVkKDEyMykKIyBSYW5kb20gZm9yZXN0IHByZWRpY3Rpb25zCnlfcHJlZCA9IHByZWRpY3QocmFuZG9tZiwgbmV3ZGF0YSA9IHRlc3RbLTEyXSkKcmFuZGZfY29uZiA8LSBjb25mdXNpb25NYXRyaXgoeV9wcmVkLCB0ZXN0JENodXJuLCBtb2RlID0gImV2ZXJ5dGhpbmciLCBwb3NpdGl2ZSA9ICIxIikKaW1wb3J0YW5jZShyYW5kb21mKQpyYW5kZl9jb25mCgpyYW5kZl9mIDwtIHJhbmRmX2NvbmYkYnlDbGFzc1s3XQpgYGAKCmBgYHtyfQpzZXQuc2VlZCgxMjMpCiMgU2ltcGxlIGRlY2lzaW9uIHRyZWVlCmR0cmVlIDwtIHJwYXJ0KGZvcm11bGEgPSBhcy5mb3JtdWxhKGZvcm11bGEpLCBkYXRhID0gdHJhaW4sIGNwPTAuMDEpCiMgQ2hvb3Npbmcgb3B0aW1hbCBjb21wbGV4aXR5IHBhcmFtZXRlciAoY3ApIHVzaW5nIHBsb3QKcHJpbnRjcChkdHJlZSkKcGxvdGNwKGR0cmVlKQoKYGBgCgpgYGB7cn0KcnBhcnQucGxvdChkdHJlZSkKYGBgCgpgYGB7cn0Kc2V0LnNlZWQoMTIzKQojIERldmlzaW9uIHRyZWUgcHJlZGljdGlvbnMKdHJlZV9wcmVkIDwtIHByZWRpY3QoZHRyZWUsIHRlc3QsIHR5cGUgPSAnY2xhc3MnKQoKZHRyZWVfY29uZiA8LSBjb25mdXNpb25NYXRyaXgodHJlZV9wcmVkLCB0ZXN0JENodXJuLCAgbW9kZSA9ICJldmVyeXRoaW5nIiwgcG9zaXRpdmUgPSAiMSIpCmR0cmVlX2NvbmYKZHRyZWVfZiA8LSBkdHJlZV9jb25mJGJ5Q2xhc3NbN10KYGBgCgpgYGB7cn0KIyBQcmVwYXJpbmcgZGF0YSBmb3IgeGdib29zdCBhbGdvcml0aG0KeGdiX3RyYWluX2NodXJuIDwtIGFzLm1hdHJpeCh0cmFpblsxMl0pCnhnYl90cmFpbiA8LSBkYXRhLm1hdHJpeCh0cmFpblstMTJdKQoKeGdiX3Rlc3RfY2h1cm4gPC0gYXMubWF0cml4KHRlc3RbMTJdKQp4Z2JfdGVzdCA8LSBkYXRhLm1hdHJpeCh0ZXN0Wy0xMl0pCgp4Z2JfdHJhaW5fbSA9IHhnYi5ETWF0cml4KGRhdGE9eGdiX3RyYWluLCBsYWJlbD14Z2JfdHJhaW5fY2h1cm4pCnhnYl90ZXN0X20gPSB4Z2IuRE1hdHJpeChkYXRhPXhnYl90ZXN0LCBsYWJlbD14Z2JfdGVzdF9jaHVybikKYGBgCgpgYGB7cn0KcGFyYW1zIDwtIGxpc3QoYm9vc3RlciA9ICJnYnRyZWUiLCBvYmplY3RpdmUgPSAiYmluYXJ5OmxvZ2lzdGljIiwgZXRhPTAuMywgZ2FtbWE9MCwgbWF4X2RlcHRoPTYsIG1pbl9jaGlsZF93ZWlnaHQ9MSwgc3Vic2FtcGxlPTEsIGNvbHNhbXBsZV9ieXRyZWU9MSkKYGBgCgpgYGB7cn0KIyBFeHRyYWN0aW5nIG9wdGltYWwgZmVhdHVyZXMgZm9yIHhnYm9vc3QgdXNpbmcgY3Jvc3MgdmFsaWRhdGlvbgp4Z2JjdiA8LSB4Z2IuY3YoIHBhcmFtcyA9IHBhcmFtcywgZGF0YSA9IHhnYl90cmFpbl9tLCBucm91bmRzID0gMTAwLCBuZm9sZCA9IDUsIHNob3dzZCA9IFQsIHN0cmF0aWZpZWQgPSBULCBwcmludC5ldmVyeS5uID0gMTAsIGVhcmx5LnN0b3Aucm91bmQgPSAyMCwgbWF4aW1pemUgPSBUKQpgYGAKCmBgYHtyfQojIFhHQm9vc3QgYWxnb3JpdGhtCnNldC5zZWVkKDEyMykKeGdiIDwtIHhnYm9vc3QoZGF0YSA9IHhnYl90cmFpbl9tLCBwYXJhbXM9cGFyYW1zLCBucm91bmRzPTQxKQpgYGAKCmBgYHtyfQpzZXQuc2VlZCgxMjMpCiMgeGdib29zdCBwcmVkaWN0aW9ucwp4Z2JfcCA9IHByZWRpY3QoeGdiLCB4Z2JfdGVzdF9tKQp4Z2JfcCA9IGFzLmZhY3Rvcihyb3VuZCh4Z2JfcCkpCgp4Z2JfY29uZiA8LSBjb25mdXNpb25NYXRyaXgoeGdiX3AsIHRlc3QkQ2h1cm4sICBtb2RlID0gImV2ZXJ5dGhpbmciLCBwb3NpdGl2ZSA9ICIxIikKeGdiX2NvbmYKeGdiX2YgPC0geGdiX2NvbmYkYnlDbGFzc1s3XQpgYGAKCmBgYHtyfQojIEltcG9ydGFuY2Ugb2YgZmVhdHVyZXMKeGdiLmltcG9ydGFuY2UobW9kZWw9eGdiKQoKIyBTSGFQIHZhbHVlcyBmb3IgZWFjaCBmZWF0dXJlCnNoYXAgPC0gc2hhcC5wcmVwKHhnYl9tb2RlbCA9IHhnYiwgWF90cmFpbiA9IHhnYl90cmFpbikKc2hhcC5wbG90LnN1bW1hcnkoc2hhcCkKYGBgCgpgYGB7cn0Kc2hhcC5wbG90LmRlcGVuZGVuY2Uoc2hhcCwgIkRheS5NaW51dGVzIiwgYWxwaGEgPSAwLjcsIGppdHRlcl93aWR0aCA9IDAuMSkKc2hhcC5wbG90LmRlcGVuZGVuY2Uoc2hhcCwgIkRheS5NaW51dGVzIiwgY29sb3JfZmVhdHVyZSA9ICdFdmVuaW5nLk1pbnV0ZXMnLCBhbHBoYSA9IDAuNywgaml0dGVyX3dpZHRoID0gMC4xKQpzaGFwLnBsb3QuZGVwZW5kZW5jZShzaGFwLCAiQ3VzdG9tZXIuU2VydmljZS5DYWxscyIsIGNvbG9yX2ZlYXR1cmUgPSAnRGF5Lk1pbnV0ZXMnLCBhbHBoYSA9IDAuNywgaml0dGVyX3dpZHRoID0gMC4xKQpgYGAKCgpgYGB7cn0Kc2hhcC5wbG90LmRlcGVuZGVuY2Uoc2hhcCwgIkludGVybmF0aW9uYWwuTWludXRlcyIsIGNvbG9yX2ZlYXR1cmUgPSAnYXV0bycsIGFscGhhID0gMC43LCBqaXR0ZXJfd2lkdGggPSAwLjEpCnNoYXAucGxvdC5kZXBlbmRlbmNlKHNoYXAsICJFdmVuaW5nLk1pbnV0ZXMiLCBjb2xvcl9mZWF0dXJlID0gJ2F1dG8nLCBhbHBoYSA9IDAuNywgaml0dGVyX3dpZHRoID0gMC4xKQpgYGAKCgpgYGB7cn0Kc2V0LnNlZWQoMTIzKQojIFByZWRpY3RpbmcgY2h1cm4gd2l0aCBlYWNoIG1vZGVsCmxvZ19wcmVkIDwtIHByZWRpY3QobG9naXQsIG5ld2RhdGEgPSBkYXRhX2VucywgdHlwZSA9ICdyZXNwb25zZScpCmxvZ19wcmVkIDwtIGFzLmZhY3RvcihpZmVsc2UobG9nX3ByZWQgPiAwLjUsMSwwKSkKCnN2bV9wcmVkIDwtIHByZWRpY3Qoc3ZtX20sIG5ld2RhdGEgPSBkYXRhX2VucykKCnJhbmRmX3ByZWQgPC0gcHJlZGljdChyYW5kb21mLCBuZXdkYXRhID0gZGF0YV9lbnMpCgpkdHJlZV9wcmVkIDwtIHByZWRpY3QoZHRyZWUsIG5ld2RhdGEgPSBkYXRhX2VucywgdHlwZSA9ICdjbGFzcycpCgp4Z2JfbWF0cml4IDwtIHhnYi5ETWF0cml4KGRhdGE9ZGF0YS5tYXRyaXgoZGF0YV9lbnNbLC0xMl0pLCBsYWJlbD1hcy5tYXRyaXgoZGF0YV9lbnMkQ2h1cm4pKQoKeGdiX3ByZWQgPC0gcHJlZGljdCh4Z2IsIG5ld2RhdGEgPSB4Z2JfbWF0cml4KQp4Z2JfcHJlZCA8LSBhcy5mYWN0b3Iocm91bmQoeGdiX3ByZWQpKQoKIyBCdWlsZGluZyB0aGUgZGF0YWZyYW1lIGZvciBlbnNlbWJsZSBtZXRob2QKcHJlZGljdGlvbnMgPC0gZGF0YS5mcmFtZShsb2dfcHJlZCwgc3ZtX3ByZWQsIHJhbmRmX3ByZWQsIGR0cmVlX3ByZWQsIHhnYl9wcmVkKQpwcmVkaWN0aW9ucyRlbnMgPC0gTkEKYGBgCgpgYGB7cn0KIyBBc3NpZ25pbmcgc2NvcmVzIGZvciBlYWNoIG1vZGVsIGJhc2VkIG9uIHJlc3BlY3RpdmUgRi1zY29yZQpzdW1fZiA8LSBzdW0obG9naXRfZiwgc3ZtX2YsIGR0cmVlX2YsIHJhbmRmX2YsIHhnYl9mKQpsb2dfc2NvcmUgPC0gKGxvZ2l0X2YgLyBzdW1fZikKc3ZtX3Njb3JlIDwtIChzdm1fZiAvIHN1bV9mKQpyYW5kZl9zY29yZSA8LSAocmFuZGZfZiAvIHN1bV9mKQpkdHJlZV9zY29yZSA8LSAoZHRyZWVfZiAvIHN1bV9mKQp4Z2Jfc2NvcmUgPC0gKHhnYl9mIC8gc3VtX2YpCgpzY29yZXMgPC0gYyhsb2dfc2NvcmUsIHN2bV9zY29yZSwgcmFuZGZfc2NvcmUsIGR0cmVlX3Njb3JlLHhnYl9zY29yZSkKYGBgCgpgYGB7cn0KIyBFbnNlbWJsZSBwcmVkaWN0aW9uIGZ1bmN0aW9uCmZvcihpIGluIDE6bGVuZ3RoKGxvZ19wcmVkKSl7CiAgeSA9IDAKICBuID0gMAogIGZvciAoaiBpbiAxOjUpewogICAgaWYgKHByZWRpY3Rpb25zW2ksIGpdID09IDApewogICAgICBuID0gbiArIHNjb3Jlc1tqXQogICAgfWVsc2V7CiAgICAgIHkgPSB5ICsgc2NvcmVzW2pdCiAgICB9CiAgfQogIGlmICh5ID4gbil7CiAgICBwcmVkaWN0aW9ucyRlbnNbaV0gPSAxCiAgfWVsc2V7CiAgICBwcmVkaWN0aW9ucyRlbnNbaV0gPSAwCiAgfQp9CmBgYAoKYGBge3J9CiMgTWVhc3VyaW5nIHRoZSBwZXJmb3JtYW5jZSBvZiBhbiBlbnNlbWJsZQplbnNfY29uZiA8LSBjb25mdXNpb25NYXRyaXgoYXMuZmFjdG9yKHByZWRpY3Rpb25zJGVucyksIGFzLmZhY3RvcihkYXRhX2VucyRDaHVybiksIG1vZGUgPSAnZXZlcnl0aGluZycsIHBvc2l0aXZlID0gJzEnKQplbnNfY29uZgplbnNfZiA8LSBlbnNfY29uZiRieUNsYXNzWzddCgpmX3Njb3Jlc19hbGwgPC0gYyhsb2dpdF9mLCBzdm1fZiwgZHRyZWVfZiwgcmFuZGZfZiwgeGdiX2YsIGVuc19mKQpmX3Njb3Jlc19hbGwKYGBgCgpgYGB7cn0KIyBNaW4gbWF4IG5vcm1hbGl6YXRpb24gZnVuY3Rpb24KbWlubWF4X25vcm0gPC0gZnVuY3Rpb24oeCkgewogICh4IC0gbWluKHgpKSAvIChtYXgoeCkgLSBtaW4oeCkpCn0KYGBgCgpgYGB7cn0KIyBEYXRhZnJhbWUgY29uc2lzdGluZyBvbmx5IG9mIHBlb3BsZSB3aG8gY2h1cm5lZApjaHVybl9kZiA8LSBkYXRhX2Vuc1tkYXRhX2VucyRDaHVybj09MSwgXQoKY2h1cm5fZGYgPC0gY2h1cm5fZGZbLCBjKCJEYXkuTWludXRlcyIsICJJbnRlcm5hdGlvbmFsLk1pbnV0ZXMiLCAiQ3VzdG9tZXIuU2VydmljZS5DYWxscyIpXQoKIyBjaHVybl9kZiA8LSBhcy5kYXRhLmZyYW1lKGxhcHBseShjaHVybl9kZiwgbWlubWF4X25vcm0pKQoKIyBjaHVybl9kZiA8LSBjaHVybl9kZlssIShuYW1lcyhjaHVybl9kZikgJWluJSBjKCJQbGFuLlR5cGUiLCAiR2VuZGVyIiwgIkNodXJuIikpXQoKY2h1cm5fZGYgPC0gYXMuZGF0YS5mcmFtZShzY2FsZShjaHVybl9kZikpCmBgYAoKYGBge3J9CiMgcGNhX3JlcyA9IHByY29tcChjaHVybl9kZiwgY2VudGVyID0gVFJVRSwgc2NhbGUgPSBUUlVFKQojIHN1bW1hcnkocGNhX3JlcykKYGBgCgoKYGBge3J9CiMgVmlzdWFsaXppbmcgZWxib3cgbWV0aG9kCgp3c3MgPC0gc2FwcGx5KDE6MTAsIGZ1bmN0aW9uKGspe2ttZWFucyhjaHVybl9kZiwgaywgbnN0YXJ0PTI1LGl0ZXIubWF4ID0gNTAgKSR0b3Qud2l0aGluc3N9KQoKcGxvdCgxOjEwLCB3c3MsCiAgICAgdHlwZT0iYiIsIHBjaCA9IDE5LCBmcmFtZSA9IEYsCiAgICAgeGxhYj0iTnVtYmVyIG9mIGNsdXN0ZXJzIEsiLAogICAgIHlsYWI9IlRvdGFsIHdpdGhpbi1jbHVzdGVycyBzdW0gb2Ygc3F1YXJlcyIpCmBgYAoKYGBge3J9CiMgRGVmaW5pbmcgb3B0aW1hbCBudW1iZXIgb2YgayBmb3Igay1tZWFucyBhbGdvcml0aG0Kc2V0LnNlZWQoMTIzKQojIFNpbGhvdWV0dGUgbWV0aG9kCmZ2aXpfbmJjbHVzdChjaHVybl9kZiwga21lYW5zLCBuc3RhcnQgPSAyNSwgay5tYXggPSAxNSwgaXRlci5tYXggPSA1MCwgbWV0aG9kID0gInNpbGhvdWV0dGUiKSsKICBsYWJzKHN1YnRpdGxlID0gIlNpbGhvdWV0dGUgbWV0aG9kIikKCmBgYAoKYGBge3J9CiMgSy1tZWFucyB3aXRoIGs9NAprNCA8LSBrbWVhbnMoY2h1cm5fZGYsIGNlbnRlcnMgPSA0LCBpdGVyLm1heCA9IDUwLCBuc3RhcnQgPSAyNSkKazQKYGBgCmBgYHtyfQojIEV2YWx1YXRpbmcgYnVpbHQgY2x1c3RlcnMgd2l0aCBzaWxob3VldHRlIHdpZHRoCnNpbCA8LSBzaWxob3VldHRlKGs0JGNsdXN0ZXIsIGRpc3QoY2h1cm5fZGYpKQpmdml6X3NpbGhvdWV0dGUoc2lsKQpgYGAKCmBgYHtyfQpjaHVybl9kZiRjbHVzdGVyID0gZmFjdG9yKGs0JGNsdXN0ZXIpCgpwIDwtIHBsb3RfbHkoY2h1cm5fZGYsIHg9fkRheS5NaW51dGVzLCB5PX5JbnRlcm5hdGlvbmFsLk1pbnV0ZXMsIAp6PX5DdXN0b21lci5TZXJ2aWNlLkNhbGxzLCBjb2xvcj1+Y2x1c3RlcikgJT4lIGFkZF9tYXJrZXJzKHNpemU9MS41KQpwcmludChwKQpgYGAKCg==